package edu.whu.irlab.woscrawl.pubmed;

import edu.whu.irlab.woscrawl.util.WebDriverFactory;
import org.apache.commons.io.FileUtils;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.ui.Select;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by zhongwei on 2017/3/3.
 * 任务在pubmed/link文件夹。
 * 结果存在pubmed/result文件夹，文件名就是2242条结果的排序number。
 */

public class CrawlId {

    public static void main(String[] args) throws IOException {
        // 替换过期的session，主要是url里面的qid和SID两个参数
        new SessionCreator().start();

        //这里测试用的，只抓了linksTo_1.txt里面的任务，需要改一下linksTo_1.txt到linksTo_45.txt
        String content = FileUtils.readFileToString(new File("pubmed/link/linksTo_1.txt"));
        // 记录之间用三个空行隔开的
        String[] arr = content.split("\n\n\n");
        // 开一个主窗口，主窗口显示一个文章被引用链接点进去的页面，然后再开6个窗口去抓每一条详情页里面的PMID。
        ExecutorService executorService = Executors.newFixedThreadPool(1);
        for (int i = 0; i < arr.length; i++) {
            // 这里的latch主要用来同步，每次处理完linksTo_XX.txt里面一条记录，就把这条记录在txt中删除。接着继续
            CountDownLatch latch = new CountDownLatch(1);
            // 回头这里也要改，现在就只抓linksTo_1.txt的。
            File file = new File("pubmed/link/linksTo_1.txt");
            String content1 = FileUtils.readFileToString(file);
            String[] arr1 = content1.split("\n\n\n");
            String task = arr1[i];
            content1 = content1.replace(task + "\n\n\n", "");
            executorService.execute(new InnerWorker(task, latch));
            try {
                latch.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            // 等这条任务完成，就把任务从txt里面删除。
            FileUtils.write(file, content1);
        }
        executorService.shutdown();
    }

    static class InnerWorker extends Thread {
        private String task;
        private WebDriver driver;
        private CountDownLatch latch;

        public InnerWorker(String task, CountDownLatch latch) {
            this.task = task;
            this.latch = latch;
        }

        @Override
        public void run() {
            // 工厂获取WebDriver，mac下面直接new ChromeDriver()就行了，但是Win上比较复杂，需要指定firefox.exe位置或者chrome.exe的位置。具体可以看看WebDriverFactory这个类。
            // 有一个问题需要注意，mac下Chrome打开浏览器，点击链接等操作是阻塞的，线程会阻塞等待链接打开，但是Windows下面的火狐浏览器好像是不阻塞的，点了链接，线程会继续执行下一条语句，因此可能需要在点了链接之后做暂停，等待浏览器渲染完。不然可能会报错。最好用谷歌浏览器，但是WebDriver好像对Windows下面的谷歌浏览器支持不好。。。
            driver = WebDriverFactory.getWebDriver();
            try {
                work();
            } catch (IOException e) {
                e.printStackTrace();
            }
            driver.quit();
            latch.countDown();
        }

        public void work() throws IOException {
            String[] arr = task.split("\n");
            String[] docIdArr = arr[0].split("=");
            String docId = docIdArr[docIdArr.length - 1];
            File folder1 = new File("pubmed");
            if (!folder1.exists())
                folder1.mkdir();
            File folder2 = new File(folder1, "result");
            if (!folder2.exists())
                folder2.mkdir();
            File file = new File(folder2, "" + docId + ".txt");
            FileWriter fileWriter = new FileWriter(file, true);
            String url = arr[2];
            runOneTask(fileWriter, url);
            fileWriter.close();
        }

        public void runOneTask(FileWriter fileWriter, String url) throws IOException {
            driver.get(url);
            driver.findElement(By.id("OpenAccess")).click();
            while (true) {
                try {
                    WebElement ele = driver.findElement(By.cssSelector("#OpenAccess_2"));
                    ele.click();
                    driver.findElement(By.id("OpenAccess_tr")).findElement(By.className("button3")).click();
                    break;
                } catch (Exception ex) {
                    // 报异常捕捉住，继续重试，直到正常
                }
            }
            // 每页显示50条
            driver.findElement(By.id("s2id_selectPageSize_.bottom")).click();
            Select select = new Select(driver.findElement(By.id("selectPageSize_.bottom")));
            List<WebElement> options = select.getOptions();
            options.get(options.size() - 1).click();

            // 开6个浏览器窗口抓取详情页的PMID。
            ExecutorService executorService = Executors.newFixedThreadPool(6);
            while (true) {
                List<WebElement> lists = driver.findElements(By.className("smallV110"));
                Set<String> stringSet = new HashSet<String>();
                for (WebElement ele : lists) {
                    stringSet.add(ele.getAttribute("href"));
                }
                final List<String> pubMedIds = new ArrayList<String>();
                // latch做同步，处理完一页50条再继续下一页。
                final CountDownLatch latch = new CountDownLatch(stringSet.size());
                for (final String str : stringSet) {
                    executorService.execute(new Runnable() {
                        public void run() {
                            WebDriver driver = WebDriverFactory.getWebDriver();
                            driver.get(str);
                            String content = driver.getPageSource();
                            Pattern pattern = Pattern.compile("PubMed ID:</span>\n" +
                                    "<value>(.+?)</value>");
                            Matcher matcher = pattern.matcher(content);
                            if (matcher.find()) {
                                pubMedIds.add(matcher.group(1));
                            }
                            driver.quit();
                            // 处理完一条
                            latch.countDown();
                        }
                    });
                }
                try {
                    // 主线程阻塞等待所有任务完成。
                    latch.await();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                // 一页所有任务完成，开始往txt写PMID。
                for (String str : pubMedIds) {
                    fileWriter.write(str + "\n");
                }
                fileWriter.flush();

                // 翻页
                WebElement next = driver.findElement(By.className("paginationNext"));
                if (next == null)
                    break;
                if (next.getAttribute("title").equals("下一页(不活动)"))
                    break;
                next.click();
            }
            executorService.shutdown();
        }
    }
}
